import pandas as pd
import Core.Gadget as Gadget
import Core.MongoDB as MongoDB
import sys
import re
import numpy as np
import Core.IO as IO
import datetime
import math
import statsmodels.api as sm
import matplotlib.pyplot as plt
import matplotlib.dates as mdate
import time

def BigPlayerList(key):
    bigPlayerList = []
    if key == "社保":
        bigPlayerList += ["社保基金"]
    if key == "证金":
        bigPlayerList +=["中证金融资产管理计划","中国证券金融"]
    if key == "外管局" :
        bigPlayerList += ["梧桐树", "北京凤山", "北京坤藤"]
    if key == "汇金":
        bigPlayerList.append("中央汇金")
    if key == "国家队基金":
        bigPlayerList += ["招商丰庆", "易方达瑞惠", "南方消费活力", "嘉实新机遇", "华夏新经济"]
    if key == "私募":
        bigPlayerList += ["淡水泉", "上海高毅", "永安国富", "景林", "星石", "上海重阳"]
    if key == "外资":
        bigPlayerList += ["香港中央结算"]
    return bigPlayerList

def StockList():
    trades = database.find("Instruments", "Stock")

    symbols = []
    for trade in trades:
        symbol = trade["Symbol"]
        symbols.append(symbol)
    return symbols

def GetPlayers(database,datetime1, datetime2, datetime3):
    datetime_cal = Gadget.ToUTCDateTime(datetime3)
    datetime1 = IO.ToDateTimeString3(datetime1)
    datetime2 = IO.ToDateTimeString3(datetime2)
    datetime3 = IO.ToDateTimeString3(datetime3)

    j = 0
    updated_stocks = 0
    df = pd.DataFrame(columns=('Name','Stocks'))  # 生成空的pandas表

    dt = Gadget.ParseDateTime(datetime1)
    instruments = Gadget.FindListedInstrument(database, dt)
    #symbols =["600762.SH", "000001.SZ", "000002.SZ","000005.SZ"]
    players = {}

    for instrument in instruments:
        symbol = instrument["Symbol"]
        if symbol == "300433.SZ":
            debug = 1
        error = [symbol]
        fundamentals = database.find("Fundamental", symbol + "_Fundamental", query={"ReportDate": datetime3})
        fundamentals_lastreport = database.find("Fundamental", symbol + "_Fundamental", query={"ReportDate": datetime2})
        if len(fundamentals) == 0 or len(fundamentals_lastreport) == 0:
            continue

        fundamental = fundamentals[0]
        updated_stocks += 1
        fundamental_lastreport = fundamentals_lastreport[0]
        datetime_before = datetime_cal - datetime.timedelta(days=100)
        quotes = database.find("Quote", symbol + "_Time_86400_Bar", datetime_before, datetime_cal)
        if len(quotes) == 0:
            continue
        quote = quotes[-1]
        price = quote["Close"]

        if "FreeFloatShareHolders" in fundamental:
            holders = fundamental["FreeFloatShareHolders"]
        else:
            error.append("季报中没有前十大股东信息")
            #print(error)
            continue

        if "FreeFloatShareHolders" in fundamental_lastreport:
            holders_lastreport = fundamental_lastreport["FreeFloatShareHolders"]
        else:
            error.append("上季报中没有前十大股东信息")
            #print(error)
            continue

        check_repeat_holders = []
        for i in range(len(holders)):
            player = holders[i]["HolderName"]
            if player not in check_repeat_holders:
                check_repeat_holders.append(player)
            else:
                players[player][symbol]["Shares"] += holders[i]["Amount"]
                players[player][symbol]["Shares_lastreport"] = 0
                players[player][symbol]["Ratio"] += holders[i]["Ratio"]
                players[player][symbol]["Ratio_lastreport"] = 0
                for i in range(len(holders_lastreport)):
                    if player == holders_lastreport[i]["HolderName"]:
                        players[player][symbol]["Shares_lastreport"] += holders_lastreport[i]["Amount"]
                        players[player][symbol]["Ratio_lastreport"] += holders_lastreport[i]["Ratio"]
                        players[player][symbol]["Change_in_shares"] = players[player][symbol]["Shares"] - \
                                                                      players[player][symbol]["Shares_lastreport"]
                        players[player][symbol]["Change_in_values"] = players[player][symbol][
                                                                          "Change_in_shares"] * price
                        players[player][symbol]["Change_in_ratio"] = players[player][symbol]["Ratio"] - \
                                                                     players[player][symbol]["Ratio_lastreport"]

            if player not in players:
                stock = {}
                doc = {}
                doc["Shares"] = holders[i]["Amount"]
                doc["Shares_lastreport"] = 0
                doc["Ratio"] = holders[i]["Ratio"]
                doc["Ratio_lastreport"] = 0
                for i in range(len(holders_lastreport)):
                    if player == holders_lastreport[i]["HolderName"]:
                        doc["Shares_lastreport"]= holders_lastreport[i]["Amount"]
                        doc["Ratio_lastreport"] = holders_lastreport[i]["Ratio"]
                        continue
                doc["Change_in_shares"] = doc["Shares"] - doc["Shares_lastreport"]
                doc["Change_in_values"] = doc["Change_in_shares"] * price
                doc["Change_in_ratio"] = doc["Ratio"]- doc["Ratio_lastreport"]
                if doc["Shares_lastreport"] == 0:
                    doc["info"] = "新增大股东"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] > 0:
                    doc["info"] = "大股东增持"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] < 0:
                    doc["info"] = "大股东减持"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] == 0:
                    doc["info"] = "大股东无操作"
                else:
                    doc["info"] = "错误信息"
                stock[symbol] = doc
                players[player] = stock
                #print(player)
                j += 1
            if symbol not in players[player]:
                doc = {}
                doc["Shares"] = holders[i]["Amount"]
                doc["Shares_lastreport"] = 0
                doc["Ratio"] = holders[i]["Ratio"]
                doc["Ratio_lastreport"] = 0
                for i in range(len(holders_lastreport)):
                    if player == holders_lastreport[i]["HolderName"]:
                        doc["Shares_lastreport"] = holders_lastreport[i]["Amount"]
                        doc["Ratio_lastreport"] = holders_lastreport[i]["Ratio"]
                        continue
                doc["Change_in_shares"] = doc["Shares"] - doc["Shares_lastreport"]
                doc["Change_in_values"] = doc["Change_in_shares"] * price
                doc["Change_in_ratio"] = doc["Ratio"] - doc["Ratio_lastreport"]
                if doc["Shares_lastreport"] == 0:
                    doc["info"] = "新增大股东"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] > 0:
                    doc["info"] = "大股东增持"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] < 0:
                    doc["info"] = "大股东减持"
                elif doc["Shares_lastreport"] != 0 and doc["Change_in_shares"] == 0:
                    doc["info"] = "大股东无操作"
                else:
                    doc["info"] = "错误信息"
                players[player][symbol] = doc

        for i in range(len(holders_lastreport)):
            player_lastreport = holders_lastreport[i]["HolderName"]
            if player_lastreport not in players:
                stock = {}
                doc = {}
                doc["Shares"] = 0
                doc["Shares_lastreport"] = holders_lastreport[i]["Amount"]
                doc["Ratio"] = 0
                doc["Ratio_lastreport"] = holders_lastreport[i]["Ratio"]
                for i in range(len(holders)):
                    if player_lastreport == holders[i]["HolderName"]:
                        continue
                doc["Change_in_shares"] = doc["Shares"] - doc["Shares_lastreport"]
                doc["Change_in_values"] = doc["Change_in_shares"] * price
                doc["Change_in_ratio"] = doc["Ratio"]- doc["Ratio_lastreport"]
                doc["info"] = "大股东退出"
                stock[symbol] = doc
                players[player] = stock
                #print(player, symbol)
                j += 1
            if symbol not in players[player]:
                doc = {}
                doc["Shares"] = 0
                doc["Shares_lastreport"] = holders_lastreport[i]["Amount"]
                doc["Ratio"] = 0
                doc["Ratio_lastreport"] = holders_lastreport[i]["Ratio"]
                for i in range(len(holders)):
                    if player_lastreport == holders[i]["HolderName"]:
                        continue
                doc["Change_in_shares"] = doc["Shares"] - doc["Shares_lastreport"]
                doc["Change_in_values"] = doc["Change_in_shares"] * price
                doc["Change_in_ratio"] = doc["Ratio"] - doc["Ratio_lastreport"]
                doc["info"] = "大股东退出"
                players[player][symbol] = doc

    print("已披露季报的公司为",updated_stocks,"家")
    #df.to_csv(r"C:\Users\zhujunheng\Desktop\players.csv", index=False)
    return players



def BigPlayers(database, datetime1, datetime2, datetime3, volume):
    players = GetPlayers(database, datetime1, datetime2, datetime3)
    print("Get Player Finished")
    increasing_holded_stocks = {}
    big_players = BigPlayerList()
    big_players_action = {}
    for player in players:
        for i in range(len(big_players)):
            if re.search( big_players[i] , player) != None:
                player_holdings = players[player]
                for stock in player_holdings:
                    if player_holdings[stock]["Change_in_values"] <= volume:
                        continue
                    if stock not in increasing_holded_stocks:
                        doc = {}
                        doc[player] = player_holdings[stock]["Change_in_values"]
                        doc["increase holdings"] = player_holdings[stock]["Change_in_values"]
                        increasing_holded_stocks[stock] = doc
                    else:
                        increasing_holded_stocks[stock][player] = player_holdings[stock]["Change_in_values"]
                        increasing_holded_stocks[stock]["increase holdings"] += player_holdings[stock][
                            "Change_in_values"]

    for stock in increasing_holded_stocks:
        players_holdings = increasing_holded_stocks[stock]
        holder_list = []
        for player in players_holdings:
            if player != "increase holdings":
                holder_list.append(player)

        print(stock + "/", increasing_holded_stocks[stock]["increase holdings"], "/", holder_list)


def holder_scanner(database,datetime1, datetime2, datetime3, bigplayerlist):
    datetime_cal = Gadget.ToUTCDateTime(datetime3)
    datetime1 = IO.ToDateTimeString3(datetime1)
    datetime2 = IO.ToDateTimeString3(datetime2)
    datetime3 = IO.ToDateTimeString3(datetime3)

    j = 0
    updated_stocks = 0
    df = pd.DataFrame(columns=('Name', 'Stocks'))  # 生成空的pandas表

    dt = Gadget.ParseDateTime(datetime1)
    instruments = Gadget.FindListedInstrument(database, dt)
    # symbols =["600762.SH", "000001.SZ", "000002.SZ","000005.SZ"]
    stock_list = {}

    for instrument in instruments:
        symbol = instrument["Symbol"]
        if symbol !="000001.SZ":
            #continue
            debug = 1
        error = [symbol]
        fundamentals = database.find("Fundamental", symbol + "_Fundamental", query={"ReportDate": datetime3})
        fundamentals_lastreport = database.find("Fundamental", symbol + "_Fundamental", query={"ReportDate": datetime2})
        if len(fundamentals) == 0 or len(fundamentals_lastreport) == 0:
            continue

        fundamental = fundamentals[0]
        updated_stocks += 1
        fundamental_lastreport = fundamentals_lastreport[0]
        datetime_before = datetime_cal - datetime.timedelta(days=100)
        quotes = database.find("Quote", symbol + "_Time_86400_Bar", datetime_before, datetime_cal)
        if len(quotes) == 0:
            continue
        quote = quotes[-1]
        price = quote["Close"]

        if "FreeFloatShareHolders" in fundamental:
            holders = fundamental["FreeFloatShareHolders"]
        else:
            error.append("季报中没有前十大股东信息")
            # print(error)
            continue

        if "FreeFloatShareHolders" in fundamental_lastreport:
            holders_lastreport = fundamental_lastreport["FreeFloatShareHolders"]
        else:
            error.append("上季报中没有前十大股东信息")
            # print(error)
            continue

        if symbol not in stock_list:
            big_players_lastreport = {}
            big_players_newreport = {}

            for i in range(10):

                for j in range(len(bigplayerlist)):
                    player_class = bigplayerlist[j]
                    player_Name = BigPlayerList(player_class)
                    for l in range(len(player_Name)):
                        if player_class not in big_players_lastreport:
                            big_players_lastreport[player_class] = 0
                        if i < len(holders_lastreport):#有些年报股东信息不足十个
                            if re.search(player_Name[l], holders_lastreport[i]["HolderName"]) != None:
                                big_players_lastreport[player_class] += holders_lastreport[i]["Amount"]

                        if player_class not in big_players_newreport:
                            big_players_newreport[player_class] = 0
                        if i < len(holders):#有些年报股东信息不足十个
                            if re.search(player_Name[l], holders[i]["HolderName"]) != None:
                                big_players_newreport[player_class] += holders[i]["Amount"]

                debug = 1
            doc = {}
            doc["lastreport"] = holders_lastreport
            doc["newreport"] = holders

            for player_Name in big_players_lastreport:
                doc[player_Name + "last_position"] = big_players_lastreport[player_Name]
                doc[player_Name + "position"] = big_players_newreport[player_Name]
                doc[player_Name + "change_in_position"] = big_players_newreport[player_Name] - big_players_lastreport[player_Name]
            stock_list[symbol] = doc

    return(stock_list)



def Backtesting(dates, no_of_stocks, initial , slip , cost , trade_detail = True , soha = True):

    symbols = StockList()
    #symbols =["600762.SH", "000001.SZ", "000002.SZ","000005.SZ"]
    Stock_info = {}
    cash = 0
    position = {}
    net = initial
    net_value_curve = []
    j = 2
    while j < len(dates):
        datetime1 = dates[j-2]
        datetime2 = dates[j-1]
        datetime3 = dates[j]
        datetime_cal = Gadget.ToUTCDateTime(datetime3)
        players = GetPlayers(database, datetime1, datetime2, datetime3)

        df = pd.DataFrame(columns=('ShareHolder', "Stock", "Change_in_holdings"))# 生成空的pandas表
        i = 0
        for player in players:  # 插入一行
            doc = players[player]
            for stock in doc:
                if doc[stock]["Change_in_values"] > 0:
                    df.loc[i] = [player, stock, doc[stock]["Change_in_values"]]
                    i += 1

        df_order_by_CIV = df.sort_values(by=["Change_in_holdings"], ascending=False)  # 从大到小排序
        top_stocks = df_order_by_CIV  # 索引需要注意：pandas前闭后开，包括前不包括后
        top_stocks = np.array(top_stocks)  # np.ndarray()
        top_stocks = top_stocks.tolist()  # list
        top_n_stocks = []
        for i in range(len(top_stocks)):
            symbol = top_stocks[i][1]
            quotes = database.find("Quote", symbol + "_Time_86400_Bar", datetime2, datetime3)
            if len(quotes) > 0:
                quote = quotes[len(quotes) - 1]
                if quote["StdDateTime"] > datetime_cal - datetime.timedelta(days=10) and len(
                        top_n_stocks) < no_of_stocks:
                    top_n_stocks.append(top_stocks[i])


        if len(position) > 0:
            for symbol in position:
                if symbol == "600762.SH":
                    alll = 1


                quotes = database.find("Quote", symbol + "_Time_86400_Bar", datetime_cal - datetime.timedelta(days=100), datetime_cal)
                quote = quotes[len(quotes) - 1]
                profit = (quote["Close"]- position[symbol]["Price"]) * position[symbol]["Volume"]
                net = net + profit - slip  * position[symbol]["Volume"] - (quote["Close"]+ position[symbol]["Price"]) * position[symbol]["Volume"]* cost
                if trade_detail:
                    print(IO.ToDateString(quote["StdDateTime"]), "Sell: " + symbol , position[symbol]["Volume"],"shares at " , quote["Close"] ,
                      "Cashflow: " , quote["Close"]* position[symbol]["Volume"], "Net Value: " , net)
            position = {}


        if len(position) == 0:
            for i in range(len(top_n_stocks)):
                symbol = top_n_stocks[i][1]
                if symbol not in position:
                    if symbol == "600762.SH":
                        alll = 1
                    quotes = database.find("Quote", symbol + "_Time_86400_Bar", datetime_cal - datetime.timedelta(days=100), datetime_cal)
                    quote = quotes[len(quotes) - 1]
                    doc = {}
                    doc["Price"] = quote["Close"]
                    if soha:
                        doc["Volume"] = math.floor((net / no_of_stocks) / (quote["Close"] * 100)) * 100
                    else:
                        doc["Volume"] = math.floor((initial / no_of_stocks) / (quote["Close"] * 100)) * 100
                    doc["Cost"] = quote["Close"] * math.floor((initial / no_of_stocks) / (quote["Close"] * 100)) * 100
                    position[symbol] = doc
                    if trade_detail:
                        print(IO.ToDateString(quote["StdDateTime"]),"Follow", top_n_stocks[i][0] , "and Buy: " + symbol, doc["Volume"], "shares at ",
                              doc["Price"],
                              "Cashflow: -", doc["Cost"], "Net Value: ", net)

        #print(IO.ToDate(quote["StdDateTime"]))
        net_value_curve.append([dates[j],net])
        j += 1
    for i in range(len(net_value_curve)):
        print(net_value_curve[i][0],net_value_curve[i][1])

def Increasing_Holded_Stocks(database, datetime1, datetime2, datetime3, volume):
    players = GetPlayers(database, datetime1, datetime2, datetime3)
    increasing_holded_stocks = {}
    newcomers = {}
    runners = {}
    decreasing_holded_stocks = {}
    for player in players:
        player_holdings = players[player]
        for stock in player_holdings:
            if player_holdings[stock]["info"] == "大股东增持":
                if stock not in increasing_holded_stocks:
                    doc = {}
                    doc[player] = player_holdings[stock]["Change_in_values"]
                    doc["Change_in_values"] = player_holdings[stock]["Change_in_values"]
                    increasing_holded_stocks[stock] = doc
                else:
                    increasing_holded_stocks[stock][player] = player_holdings[stock]["Change_in_values"]
                    increasing_holded_stocks[stock]["Change_in_values"] += player_holdings[stock]["Change_in_values"]
            elif player_holdings[stock]["info"] == "大股东减持":
                if stock not in decreasing_holded_stocks:
                    doc = {}
                    doc[player] = player_holdings[stock]["Change_in_values"]
                    doc["Change_in_values"] = player_holdings[stock]["Change_in_values"]
                    decreasing_holded_stocks[stock] = doc
                else:
                    decreasing_holded_stocks[stock][player] = player_holdings[stock]["Change_in_values"]
                    decreasing_holded_stocks[stock]["Change_in_values"] += player_holdings[stock]["Change_in_values"]
            elif player_holdings[stock]["info"] == "新增大股东":
                if stock not in newcomers:
                    doc = {}
                    doc[player] = player_holdings[stock]["Change_in_values"]
                    doc["Change_in_values"] = player_holdings[stock]["Change_in_values"]
                    newcomers[stock] = doc
                else:
                    newcomers[stock][player] = player_holdings[stock]["Change_in_values"]
                    newcomers[stock]["Change_in_values"] += player_holdings[stock]["Change_in_values"]


    for stock in increasing_holded_stocks:
        players_holdings = increasing_holded_stocks[stock]
        holder_list = []
        for player in players_holdings:
            if player != "increase holdings":
                holder_list.append(player)

        print(stock, "/",increasing_holded_stocks[stock]["increase holdings"], "/",holder_list)

    debug = 1




database = MongoDB.MongoDB("10.13.38.31", "27017")

datetime_start = datetime.datetime(2004, 5, 2)
datetime_start = Gadget.ToUTCDateTime(datetime_start)
datetime_end = datetime.datetime(2018, 10, 30)
datetime_end = Gadget.ToUTCDateTime(datetime_end)
dates = Gadget.GenerateReportDates(datetime_start,datetime_end)

datetime1 = datetime.datetime(2017, 12, 31)
datetime2 = datetime.datetime(2018, 6, 30)
datetime3 = datetime.datetime(2018, 9, 30)
bigplayerlist = ["社保","证金","外管局","汇金","国家队基金","外资","私募"]

#Increasing_Holded_Stocks(database, datetime1, datetime2, datetime3, 3000000000)#大股东增持
#GetPlayers(database,datetime1,datetime2,datetime3)#统计已更新季报的公司数量
Player_info = holder_scanner(database,datetime1, datetime2, datetime3, bigplayerlist)
print("Stock"," ",  end="")
for i in range(len(bigplayerlist)):
    print(bigplayerlist[i]+"上期持仓", bigplayerlist[i]+"本期持仓",bigplayerlist[i]+"持仓变化"," ", end = "")
print("")
for symbol in Player_info:
    print(symbol," ", end = "")
    for i in range(len(bigplayerlist)):
        print(Player_info[symbol][bigplayerlist[i]+ "last_position"],Player_info[symbol][bigplayerlist[i]+"position"],Player_info[symbol][bigplayerlist[i]+"change_in_position"], " ", end="")
        if i == len(bigplayerlist)-1:
            print("")







#BigPlayers(database, datetime1, datetime2, datetime3, 0)#国家队增持

#print(sys.getdefaultencoding())
#Backtesting(dates, 10, 100000 , 0.01 , 0.0005)